revive support for ENCODING in xcsv files.

diff --git a/csv_util.cc b/csv_util.cc
index 3304ae4a5ee38547f610b74a9babb61ab4858d98..78c705d1641c857964b7f27abf23ada0199b79ed 100644 (file)
--- a/csv_util.cc
+++ b/csv_util.cc
@@ -21,9 +21,9 @@
  */
 
 #include <QtCore/QRegExp>
+#include <QtCore/QTextStream>
 
 #include "defs.h"
-#include "cet_util.h"
 #include "csv_util.h"
 #include "garmin_fs.h"
 #include "grtcirc.h"
@@ -730,7 +730,9 @@ void xcsv_file_init(void)
   xcsv_file.badchars = QString();
   xcsv_file.ifield_ct = 0;
   xcsv_file.ofield_ct = 0;
-  xcsv_file.xcsvfp = NULL;
+  xcsv_file.file = NULL;
+  xcsv_file.stream = NULL;
+  xcsv_file.codec = NULL;
   xcsv_file.fname = QString();
   xcsv_file.description = NULL;
   xcsv_file.extension = NULL;
@@ -1406,6 +1408,26 @@ xcsv_parse_val(const char* s, Waypoint* wpt, const field_map_t* fmp,
   }
 }
 
+// TODO: eliminate this routine which is modeled
+// after gbfgetstr for legacy compatibility.
+static char*
+xcsv_readline(char* buff)
+{
+  if (buff) {
+    xfree(buff);
+  }
+  QString line = xcsv_file.stream->readLine();
+  if (line.isNull()) {
+    return NULL;
+  } else {
+    // TODO: move csv processing to Qt, eliminating the need to go
+    // back to 8 bit encoding, which is shaky for encoding like utf8
+    // that have multibyte characters.
+    char* newbuff = xstrdup(CSTR(line));
+    return newbuff;
+  }
+}
+
 /*****************************************************************************/
 /* xcsv_data_read() - read input file, parsing lines, fields and handling    */
 /*                   any data conversion (the input meat)                    */
@@ -1413,7 +1435,7 @@ xcsv_parse_val(const char* s, Waypoint* wpt, const field_map_t* fmp,
 void
 xcsv_data_read(void)
 {
-  char* buff;
+  char* buff = NULL;
   char* s;
   Waypoint* wpt_tmp;
   int linecount = 0;
@@ -1433,11 +1455,8 @@ xcsv_data_read(void)
     csv_route = rte;
   }
 
-  while ((buff = gbfgetstr(xcsv_file.xcsvfp))) {
-    if ((linecount == 0) && xcsv_file.xcsvfp->unicode) {
-      cet_convert_init(CET_CHARSET_UTF8, 1);
-    }
-
+  // TODO: stop the back and forth between QString and char strings,
+  while ((buff = xcsv_readline(buff))) {
     linecount++;
     /* Whack trailing space; leading space may matter if our field sep
      * is whitespace and we have leading whitespace.
@@ -1643,7 +1662,7 @@ xcsv_waypt_pr(const Waypoint* wpt)
     fmp = (field_map_t*) elem;
 
     if ((i != 0) && !(fmp->options & OPTIONS_NODELIM)) {
-      gbfputs(write_delimiter, xcsv_file.xcsvfp);
+      *xcsv_file.stream << write_delimiter;
     }
 
     if (fmp->options & OPTIONS_ABSOLUTE) {
@@ -2152,7 +2171,7 @@ xcsv_waypt_pr(const Waypoint* wpt)
 
     if (!xcsv_file.field_encloser.isEmpty()) {
       /* print the enclosing character(s) */
-      gbfputs(xcsv_file.record_delimiter, xcsv_file.xcsvfp);
+      *xcsv_file.stream << xcsv_file.record_delimiter;
     }
 
     /* As a special case (pronounced "horrible hack") we allow
@@ -2161,16 +2180,16 @@ xcsv_waypt_pr(const Waypoint* wpt)
     if (0 == strcmp(fmp->printfc, "\"%s\"")) {
       obuff = '"' + obuff + '"';
     }
-    gbfputs(obuff, xcsv_file.xcsvfp);
+    *xcsv_file.stream << obuff;
 
     if (!xcsv_file.field_encloser.isEmpty()) {
       /* print the enclosing character(s) */
-      gbfputs(xcsv_file.record_delimiter, xcsv_file.xcsvfp);
+      *xcsv_file.stream << xcsv_file.record_delimiter;
     }
     buff.clear();
   }
 
-  gbfputs(xcsv_file.record_delimiter, xcsv_file.xcsvfp);
+  *xcsv_file.stream << xcsv_file.record_delimiter;
 
   /* increment the index counter */
   waypt_out_count++;
@@ -2227,8 +2246,7 @@ xcsv_data_write(void)
       QString t = dt.toString("hh:mm:ss");
       cout.replace("__TIME__", t);
     }
-    gbfputs(cout, xcsv_file.xcsvfp);
-    gbfputs(xcsv_file.record_delimiter, xcsv_file.xcsvfp);
+    *xcsv_file.stream << cout <<  xcsv_file.record_delimiter;
   }
 
   if ((xcsv_file.datatype == 0) || (xcsv_file.datatype == wptdata)) {
@@ -2243,8 +2261,7 @@ xcsv_data_write(void)
 
   /* output epilogue lines, if any. */
   foreach(const QString& ogp, xcsv_file.epilogue) {
-    gbfputs(ogp, xcsv_file.xcsvfp);
-    gbfputs(xcsv_file.record_delimiter, xcsv_file.xcsvfp);
+    *xcsv_file.stream << ogp << xcsv_file.record_delimiter;
   }
 }
 #endif

diff --git a/csv_util.h b/csv_util.h
index 83bd87603ff9ac3865b216c929e98106e0c6f92f..e0e16a496c90072b8158941eb39d225589e6d91b 100644 (file)
--- a/csv_util.h
+++ b/csv_util.h
@@ -96,6 +96,10 @@ typedef struct char_map {
   const char* chars;
 } char_map_t;
 
+namespace gpsbabel
+{
+    class File;
+}
 /*
  * a Class describing all the wonderful elements of xcsv files, in a
  * nutshell.
@@ -125,7 +129,9 @@ class XcsvFile {
   int ifield_ct;               /* actual # of ifields */
   int ofield_ct;               /* actual # of ofields */
 
-  gbfile* xcsvfp;              /* ptr to current *open* data file */
+  gpsbabel::File* file;
+  QTextStream* stream;
+  QTextCodec* codec;
   QString fname;                 /* ptr to filename of above. */
 
   char* description;           /* Description for help text */

diff --git a/xcsv.cc b/xcsv.cc
index a535554e79ee1f57f447ad0be2d0b667725a5239..058a3cc97b87a253590b81a338f6401b5b9bb108 100644 (file)
--- a/xcsv.cc
+++ b/xcsv.cc
@@ -23,10 +23,14 @@
 
  */
 
+#include <QtCore/QTextCodec>
+#include <QtCore/QTextStream>
+
 #include "defs.h"
-#include "cet_util.h"
 #include "csv_util.h"
 #include "jeeps/gpsmath.h"
+#include "src/core/file.h"
+#include "src/core/logging.h"
 #include <ctype.h>
 #include <stdlib.h>
 
@@ -335,7 +339,10 @@ xcsv_parse_style_line(char* sbuff)
 
                           if (ISSTOKEN(sbuff, "ENCODING")) {
                             p = csv_stringtrim(&sbuff[8], "\"", 1);
-                            cet_convert_init(p, 1);
+                            xcsv_file.codec = QTextCodec::codecForName(p);
+                            if (!xcsv_file.codec) {
+                              Fatal() << "Unsupported character set '" << p << "'.";
+                            }
                             xfree(p);
                           } else
 
@@ -554,7 +561,16 @@ xcsv_rd_init(const QString& fname)
     }
   }
 
-  xcsv_file.xcsvfp = gbfopen(fname, "r", MYNAME);
+  xcsv_file.file = new gpsbabel::File(fname);
+  xcsv_file.file->open(QFile::ReadOnly);
+  xcsv_file.stream = new QTextStream(xcsv_file.file);
+  if (xcsv_file.codec) {
+    xcsv_file.stream->setCodec(xcsv_file.codec);
+  } else {
+    // default to UTF-8.
+    xcsv_file.stream->setCodec("UTF-8");
+    xcsv_file.stream->setAutoDetectUnicode(true);
+  }
   xcsv_file.gps_datum = GPS_Lookup_Datum_Index(opt_datum);
   is_fatal(xcsv_file.gps_datum < 0, MYNAME ": datum \"%s\" is not supported.", opt_datum);
 }
@@ -562,7 +578,12 @@ xcsv_rd_init(const QString& fname)
 static void
 xcsv_rd_deinit(void)
 {
-  gbfclose(xcsv_file.xcsvfp);
+  xcsv_file.file->close();
+  delete xcsv_file.file;
+  xcsv_file.file = NULL;
+  delete xcsv_file.stream;
+  xcsv_file.stream = NULL;
+  xcsv_file.codec = NULL;
 
   xcsv_destroy_style();
 }
@@ -586,7 +607,19 @@ xcsv_wr_init(const QString& fname)
     xcsv_read_style(styleopt);
   }
 
-  xcsv_file.xcsvfp = gbfopen(fname, "w", MYNAME);
+  xcsv_file.file = new gpsbabel::File(fname);
+  xcsv_file.file->open(QFile::WriteOnly | QFile::Text);
+  xcsv_file.stream = new QTextStream(xcsv_file.file);
+  if (xcsv_file.codec) {
+    xcsv_file.stream->setCodec(xcsv_file.codec);
+    // enable bom for all UTF codecs except UTF-8
+    if (xcsv_file.codec->mibEnum() != 106) {
+      xcsv_file.stream->setGenerateByteOrderMark(true);
+    }
+  } else {
+    // emulate gbfputs which assumes UTF-8.
+    xcsv_file.stream->setCodec("UTF-8");
+  }
   xcsv_file.fname = fname;
 
   /* set mkshort options from the command line */
@@ -624,7 +657,13 @@ xcsv_wr_position_init(const QString& fname)
 static void
 xcsv_wr_deinit(void)
 {
-  gbfclose(xcsv_file.xcsvfp);
+  xcsv_file.stream->flush();
+  xcsv_file.file->close();
+  delete xcsv_file.file;
+  xcsv_file.file = NULL;
+  delete xcsv_file.stream;
+  xcsv_file.stream = NULL;
+  xcsv_file.codec = NULL;
 
   xcsv_destroy_style();
 }
@@ -652,7 +691,7 @@ xcsv_wr_position(Waypoint* wpt)
   xcsv_data_write();
   waypt_del(wpt);
 
-  gbfflush(xcsv_file.xcsvfp);
+  xcsv_file.stream->flush();
 }
 
 ff_vecs_t xcsv_vecs = {